#include <vector>
#include <list>
#include <map>
#include <set>
#include <deque>
#include <queue>
#include <stack>
#include <bitset>
#include <algorithm>
#include <functional>
#include <numeric>
#include <iterator>
#include <utility>
#include <sstream>
#include <iostream>
#include <iomanip>
#include <cstdio>
#include <cmath>
#include <cstdlib>
#include <cctype>
#include <string>
#include <cstring>
#include <cstdio>
#include <cmath>
#include <cstdlib>
#include <ctime>
#define for_size(var, size) \
    for(int var = 0; var < (size); ++var)
#define for_range(var, from, to) \
    for(int var = (from); var < (to); ++var)
#define for_ranges(var, from, to, step) \
    for(int var = (from); var < (to); var += (step))
#define for_each(var, container) \
    for(typeof((container).begin()) var = (container).begin(); var != (container).end(); ++var)

using namespace std;

const string TR_FNAME = "training.txt";
const string TE_FNAME = "test.txt";
const int TYPE_NUM = 3;
const int TYPE_ID_DES = 0;
const int TYPE_ID_TIT = 1;
const int TYPE_ID_QRY = 2;
const int TYPE_ID_PUR = 3;
const int CHUNK_SIZE = 1000000;
const string BASE_FNAME[4] = {"descriptionid_tokensid.txt", "titleid_tokensid.txt", "queryid_tokensid.txt", "purchasedkeywordid_tokensid.txt"};

const int TOPIC_NUM = 100;
float ldaTopicProb[TOPIC_NUM];
int* ldaTopic[TYPE_NUM];
int recordNum[TYPE_NUM];
char buffer[1024];
string modelId;

int main(int argc, char* argv[]) {
    if (argc == 1) {
        printf("Usage: %s model_id\n  e.g. model_id = \"d\" if .plda.d.result exists\n", argv[0]);
        return 1;
    }
    modelId = argv[1];
    for_size(i, TYPE_NUM) {
        string ldaResultFName = "lda/" + BASE_FNAME[i] + ".plda." + modelId + ".result";
        
        fprintf(stderr, "Counting %s\n", ldaResultFName.data());
        sprintf(buffer, "wc -l %s", ldaResultFName.data());
        FILE* output = popen(buffer, "r");
        fscanf(output, "%d", &recordNum[i]);
        pclose(output);
        
        fprintf(stderr, "%d records in %s\n", recordNum[i], ldaResultFName.data());
        
        fprintf(stderr, "Reading %s\n", ldaResultFName.data());
        ldaTopic[i] = new int[recordNum[i]];
        FILE* ldaF = fopen(ldaResultFName.data(), "r"); 
        for_size(j, recordNum[i]) {
            int maxTopic = -1;
            for_size(k, TOPIC_NUM) {
                fscanf(ldaF, "%f", &ldaTopicProb[k]);
                if (maxTopic == -1 || ldaTopicProb[k] > ldaTopicProb[maxTopic]) {
                    maxTopic = k;
                }
            }
            ldaTopic[i][j] = maxTopic + 1;
        }
        fclose(ldaF);
    }
    
    fprintf(stderr, "Reading %s\n", TR_FNAME.data());
    FILE* trF = fopen(TR_FNAME.data(), "r"); 
    FILE* feaF = NULL;
    int insId = 0;
    while (true) {
        if (insId % CHUNK_SIZE == 0) {
            if (feaF != NULL) {
                fclose(feaF);
            }
            sprintf(buffer, "features/token_feature_%d.txt", insId / CHUNK_SIZE + 1);
            feaF = fopen(buffer, "w");
        }
        int click, imp, adId, aderId, depth, pos, qId, kId, tId, dId, uId; 
        if (fscanf(trF, "%d%d%s%d%d%d%d%d%d%d%d%d", &click, &imp, buffer, &adId, &aderId, &depth, &pos, &qId, &kId, &tId, &dId, &uId) == EOF)
            break;
        fprintf(feaF, "[%d,%d,%d]\n", ldaTopic[TYPE_ID_DES][dId], ldaTopic[TYPE_ID_TIT][tId], ldaTopic[TYPE_ID_QRY][qId]); 
        //fprintf(feaF, "[%d,%d]\n", ldaTopic[TYPE_ID_DES][dId], ldaTopic[TYPE_ID_TIT][tId]); 
        insId++;
    }
    fclose(trF);
    if (feaF != NULL) {
        fclose(feaF);
    }
    
    for_size(i, (insId - 1) / CHUNK_SIZE + 1) {
        sprintf(buffer, "mv features/feature_%d.txt features/old_feature_%d.txt", i + 1, i + 1);
        system(buffer);
        sprintf(buffer, "paste features/old_feature_%d.txt features/token_feature_%d.txt > features/feature_%d.txt", i + 1, i + 1, i + 1);
        system(buffer);
    }
    
    int teChunk = 151;
    fprintf(stderr, "Reading %s\n", TE_FNAME.data());
    FILE* teF = fopen(TE_FNAME.data(), "r"); 
    sprintf(buffer, "features/token_feature_%d.txt", teChunk);
    FILE* teFeaF = fopen(buffer, "w");
    while (true) {  
        int adId, aderId, depth, pos, qId, kId, tId, dId, uId; 
        if (fscanf(teF, "%s%d%d%d%d%d%d%d%d%d", buffer, &adId, &aderId, &depth, &pos, &qId, &kId, &tId, &dId, &uId) == EOF)
            break;
        fprintf(teFeaF, "[%d,%d,%d]\n", ldaTopic[TYPE_ID_DES][dId], ldaTopic[TYPE_ID_TIT][tId], ldaTopic[TYPE_ID_QRY][qId]); 
    }
    fclose(teF);
    fclose(teFeaF);
        
    sprintf(buffer, "mv features/feature_%d.txt features/old_feature_%d.txt", teChunk, teChunk);
    system(buffer);
    sprintf(buffer, "paste features/old_feature_%d.txt features/token_feature_%d.txt > features/feature_%d.txt", teChunk, teChunk, teChunk);
    system(buffer);
    
    return 0;
}

